from IPython.display import Javascript, display
from ipywidgets import widgets
def run_all(ev):
display(Javascript('IPython.notebook.execute_cells_below()'))
button = widgets.Button(description="Click to load experiment summary:")
button.on_click(run_all)
display(button)
# initializes dbconn and notebook settings
# imports relevant data from mysql db
import pandas as pd
import numpy as np
import sqlalchemy as sa
import glob
import os
import sys
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import datetime
%matplotlib inline
#%pylab inline
pylab.rcParams['figure.figsize'] = (15, 25)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import seaborn as sns
import qgrid
qgrid.nbinstall(overwrite=True)
qgrid.set_defaults(remote_js=True, show_toolbar=True)
import loaderplots as lp
from scipy.stats import stats, ttest_ind, ttest_rel, rankdata
#engine = sa.create_engine('mysql+pymysql://reeluser:R3el$olarX@localhost/rsi?charset=utf8')
engine = sa.create_engine('mysql+pymysql://reeluser:R3el$olarX@192.168.21.175/rsi?charset=utf8')
conn = engine.connect()
#cds, cells, exp, fullexp, fullprocess, ivt, ivtdfmod, mods, modtrends, process, sencera, trends, xrf
#senceradf = pd.read_sql("sencera", conn, index_col="sencera_index")
#processdf = pd.read_sql("process", conn, index_col="process_index")
#expdf = pd.read_sql("exp", conn, index_col="exp_index")
#celldf = pd.read_sql("cells", conn, index_col="cells_index")
#moddf = pd.read_sql("mods", conn, index_col="mods_index")
#%time trends = pd.read_sql("trends", conn, index_col="trends_index")
#xrf = pd.read_sql("xrf", conn, index_col="xrf_index")
#ivtdf = pd.read_sql("ivt", conn, index_col="ivt_index")
#eddf = pd.read_sql("ed", conn, index_col="ed_index")
import time
now = time.strftime("%c")
print ("Updated at: %s" % now )
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
topic = (input("Enter experimental topic/description: "))
#cursor = conn.cursor()
#cursor.execute('SELECT DISTINCT `Experiment_ID`, `Description` FROM exp WHERE `Description` OR `Split1` OR `Split2` OR `Split3` LIKE "%{0}%";', (topic))
query = 'SELECT DISTINCT Experiment_ID, Description, Split1, Split2 FROM fullexp WHERE Description LIKE "%%{0}%%" OR Split1 LIKE "%%{0}%%" OR Split2 LIKE "%%{0}%%" OR Split3 LIKE "%%{0}%%";'
query = query.format(topic)
print ("\n")
print ("List of experiments involving %s updated at %s" % (topic, now) )
topicdf = pd.read_sql_query(query, conn)
qgrid.show_grid(topicdf, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
click "export" on the table to save it for offline use when exporting as html.
expt = int(input("Enter experiment number: "))
#query = 'SELECT * from trends WHERE `Experiment_ID` = %d;' % expt
query = 'SELECT * from filttrends WHERE `Experiment_ID` = %d;' % expt
df = pd.read_sql_query(query, conn)
print ("Imported Expt. %d at %s" % (expt, now) )
#'{0} in {1}'.format(unicode(self.author,'utf-8'),unicode(self.publication,'utf-8'))
#pd.set_option('display.notebook_repr_html', True)
#def _repr_latex_(self):
# return "\centering{%s}" % self.to_latex()
#pd.DataFrame._repr_latex_ = _repr_latex_ # monkey patch pandas DataFrame
Below, ranked module performance data.
#create module df
modsdf = df[df.Cell_Count > 19]
def mod_shunted(row):
if row['Rsh_Ohm'] <=500.:
return 1
modsdf.loc[:,'shunted'] = modsdf.apply(lambda row: mod_shunted(row), axis=1)
def mod_shunt(row):
if row['Rsh'] <=50.:
return 1
modsdf.loc[:,'shunt_count'] = modsdf.apply(lambda row: mod_shunt(row), axis=1)
#create nonshunted module df
nonshuntedmods = modsdf[modsdf.loc[:,'shunted'] != 1]
modsdflight = modsdf[~modsdf['Measurement'].str.contains('.*Dark.*', na = False)]
nonshuntedlight = nonshuntedmods[~nonshuntedmods['Measurement'].str.contains('.*Dark.*', na = False)]
#flatui1 = ["#3498db","#e74c3c", "#2ecc71", "#34495e", "#95a5a6", "#9b59b6", "#3498db", "#e74c3c",]
#sns.set_palette(flatui1)
#lp.mod_por_plots(modsdf, '%s' %expt)
query = 'SELECT Split1, Split2, Split3 from fullexp WHERE `Experiment_ID` = %s;' % expt
dfs = pd.read_sql_query(query, conn)
splitexist = []
splitexist = (dfs.isnull().all())
splitdict = dict(splitexist)
splitdict
def removeit(l):
d = dict(l)
for i in d.copy(): #if not d[i] returns true (null) splits
if d[i] == True:
d.pop(i)
return d
splitlist = removeit(splitexist)
modstrimmed = modsdflight.groupby('Sample_ID', as_index=False).max()
nonshuntedtrimmed = nonshuntedlight.groupby('Sample_ID', as_index=False).max()
lp.rankedfiltered(splitlist, modstrimmed, '%s' %expt)
here is just a simple chart showing shunted modules for POR vs EXP
#lp.shunts(splitlist, modstrimmed, '%s' %expt)
we can quickly output experimental stats
grouped = modsdflight.groupby('POR', as_index=True)
#exp is 0
modstats = pd.DataFrame(grouped.describe())
#modstats = modstats.set_index(['POR'])
print("Expt %d module stats: " % expt)
#modstats = modstats.drop('trends_index', axis=1)
modstats = modstats.drop('Experiment_ID', axis=1)
modstats = modstats.drop('Cell_Area_mm2', axis=1)
modstats = modstats.drop('Cell_Count', axis=1)
modstats = modstats.drop('Cell_no', axis=1)
modstats
#qgrid.show_grid(modstats, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
plt.figure(figsize=(9,6))
ax = sns.stripplot(x="CdTe Tool", y="percentEff", data=modstrimmed, hue= "Split1", jitter=True)
sns.violinplot(x="CdTe Tool", y="percentEff", data=modstrimmed, inner=None, color=".8", ax=ax)
plt.legend(loc="lower left", frameon=True, fontsize=10)
lp.bycdtetool(splitlist, modstrimmed, '%s' %expt)
This loader takes user input. In the example below, you can choose to plot by Split1, Split2, POR, CdTe Tool, CdS Tool, or almost anything, vs. CdTe deposition time
lp.bydepplots(modstrimmed)
#lp.jointplots(modsdf, '%s' %expt)
# create cells df
cellsdf = df[df.Cell_Count == 1]
def cell_shunted(row):
if row['Rsh_Ohm'] <=500.:
return 1
cellsdf['shunted'] = cellsdf.apply(lambda row: cell_shunted(row), axis=1)
def cell_shunt(row):
if row['Rsh'] <=50.:
return 1
cellsdf['shunt_count'] = cellsdf.apply(lambda row: cell_shunt(row), axis=1)
nscells = cellsdf[cellsdf.loc[:,'shunted'] != 1]
cellsdflight = cellsdf[~cellsdf['Measurement'].str.contains('.*Dark.*', na = False)]
nscellslight = nscells[~nscells['Measurement'].str.contains('.*Dark.*', na = False)]
cellgroup = cellsdflight.groupby('POR', as_index=False)
#nscellgroup = nscellslight.groupby('POR', as_index=False)
cellstrimmed = cellsdflight.groupby(['Sample_ID','Cell_no'], as_index=False).max()
nscellstrimmed = nscellslight.groupby(['Sample_ID','Cell_no'], as_index=False).max()
lp.rankedfiltered(splitlist, cellstrimmed, '%s' %expt)
#nscells, cellsdflight, nscellslight, cellstrimmed, nscellstrimmed, cells df
#modsdf, modsdflight, modstrimmed,
#setting up grouped median and mean dfs
#modmedian = modsdf.groupby('Split1', as_index=False).median()
#modmean = modsdf.mean()
#cellmedian = cellsdf.groupby('Split1', as_index=False).median()
#cellmean = cellsdf.mean()
sns.set_palette("coolwarm")
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(figsize=(13,13), nrows=2, ncols=2)
#plt.subplot(411)
b = sns.boxplot('CdTe Tool', 'percentEff', hue='Split1', data=nonshuntedtrimmed, notch=False, ax=ax1)
# bplot = sns.boxplot('Sample_ID', 'percentEff', hue='POR', data=df, notch=False)
#plt.subplot(412)
b1 = sns.boxplot('CdTe Tool', 'percentEff', hue='Split1', data=nscellstrimmed, notch=False, ax=ax2)
#plt.subplot(421)
b2 = sns.boxplot('CdTe Tool', 'percentEff', hue='Split2', data=nonshuntedtrimmed, notch=False, ax=ax3)
#plt.subplot(422)
b3 = sns.boxplot('CdTe Tool', 'percentEff', hue='Split2', data=nscellstrimmed, notch=False, ax=ax4)
qgrid.show_grid(modstrimmed, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
qgrid.show_grid(cellstrimmed, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
lp.expbycell(cellsdf, '%s' %expt)
Supplementary modules:
(1) T-test module: takes user input (POR, Splits, CdTe tool, etc)
(2) Past 50 experiments trends
(3) Module inspector: user selects module of interest; outputs Eff by cell (table and figure) and EL image
(1) input t-testing parameter when asked
(2) returns all permutations, sorted by significance
* typically uses modsdflight, but could use modstrimmed (may have higher errors)
#the following function returns all ttests within a variable (ie Splits in Split 1, POR vs EXP, CDTE 1 vs 2)
#if instead you prefer to choose which samples to test, use: lp.ttestbychoice(df)
#df of choice is modsflight
lp.ttestall(modsdflight)
flatui = ["#3498db", "#e74c3c", "#2ecc71", "#95a5a6", "#34495e", "#9b59b6"]
sns.set(style='ticks')
sns.set_palette(flatui)
sns.set_context("poster")
#uncomment for user input
#x_low = float(input("enter beginning of trends range: "))
#x_high = float(input("enter end of trends range: "))
#previous 50 experiments
x_low = int(df.Experiment_ID.unique() - 49.)
x_high = int(df.Experiment_ID.unique()) + 1
query = 'SELECT * FROM filttrends WHERE Experiment_ID BETWEEN {0} AND {1} AND Measurement LIKE "Light" AND Cell_Count > 20 ;'
query = query.format(x_low, x_high)
trendsdf = pd.read_sql_query(query, conn)
lp.portrends(trendsdf)
(1) use the table below to choose module
(2) input Sample ID when asked
samps = []
query = 'SELECT DISTINCT `Sample_ID`, Description, percentEff, Split1, Split2, Split3 from filttrends WHERE `Cell_Count` > 20 AND `Experiment_ID` = %d AND Measurement LIKE "Light";' % expt
samps = pd.read_sql_query(query, conn)
samplist = samps["Sample_ID"].tolist()
samps2 = samps.groupby('Sample_ID', as_index=False).max()
qgrid.show_grid(samps2, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
sample = (input("Enter sample ID: ")).upper()
#cursor = conn.cursor()
#cursor.execute('SELECT DISTINCT `Experiment_ID`, `Description` FROM exp WHERE `Description` OR `Split1` OR `Split2` OR `Split3` LIKE "%{0}%";', (topic))
query = 'SELECT * FROM filttrends WHERE Sample_ID LIKE "%%{0}%%" AND Cell_Count = 1 AND Measurement LIKE "Light";'
query = query.format(sample)
sampledf = pd.read_sql_query(query, conn)
qgrid.show_grid(sampledf, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
sns.set_style("darkgrid")
fg = sns.FacetGrid(sampledf, hue='POR', size= 6, aspect=2, )#, hue_kws=dict(marker=["^", "v"]))
fg.map(plt.scatter, 'Cell_no', 'percentEff', alpha=1,s=50)
#plt.xlim(570,700)
plt.ylim(0, 18)
plt.axhline(16.7, color='r', linestyle='-.')
pins = [1, 4, 7, 10, 13, 16, 19, 22]
for xc in pins:
plt.axvline(x=xc, color = 'c', linestyle = ':', lw=2)
#plt.axvline(1, color='r', linestyle='-.')
plt.grid(b=True, which='both', color='0.65',linestyle='-', )
#plt.legend()
plt.tick_params(axis='both',which='minor')
plt.minorticks_on()
plt.title('%s eff by cell' % sample)
eldate = str(sampledf["Date_tm"].unique())
eldate = eldate[2:12]
#eldate = eldate[7:11]+"-0"+eldate[2:3]+"-"+eldate[4:6]
#print(eldate)
from IPython.display import Image
PATH = '//rsi-el/Users/Public/EL/'
#filename =PATH + "%s/%s.jpg" % (eldate,sample)
#print(filename)
Image(filename = PATH + "%s/%s.jpg" % (eldate,sample), width=700, height=700)